package engine

import (
    "log"
    "spider/simple/fetcher"
    "spider/simple/parser"
)

func Run(seeds ...parser.Request) {
    var requests []parser.Request
    
    requests = append(requests, seeds...)
    
    for len(requests) > 0 {
        // 相当于队列pop操作
        request := requests[0]
        requests = requests[1:]
        
        log.Printf("Fetching %s", request.Url)
        
        //  获取内容
        content, err := fetcher.Fetch(request.Url)
        if err != nil {
            log.Printf("Fetcher: error fetching url %s : %v", request.Url, err)
            continue
        }
        
        // 解析内容
        parseResult := request.Parser.Analysis(content)
        
        // 将解析出来的request追加到request队列等待继续执行
        requests = append(requests, parseResult.Requests...)
        
        // 打印结果
        for _, value := range parseResult.ReponseSlice {
            log.Printf("got result %v", value)
        }
    }
}
